# -*- coding: utf-8 -*-
import scrapy
import re
import datetime
from scrapy.http import Request
from urllib import parse
from scrapy.loader import ItemLoader

from ArticleSpider.items import JobBoleArticleItem, ArticleItemLoader
from ArticleSpider.utils.common import get_md5


class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    # start_urls = ['http://blog.jobbole.com/']
    # start_urls = ['http://blog.jobbole.com/113568/']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    def parse(self, response):
        """
        1. 获取文章列表页中的文章 url 并交给 scrapy 下载后给解析函数进行具体字段的解析
        2. 获取下一页的 url 并交给 scrapy 下载，下载完成后交给parse函数
        :param response:
        :return:
        """
        # 解析列表页中的所有文章 url 并交给 scrapy 下载后解析
        post_nodes = response.css("#archive .floated-thumb .post-thumb a")

        for post_node in post_nodes:  # 这里提取出了url 下一步 下载 下一步 交给自己的解析函数
            image_url = post_node.css("img::attr(src)").extract_first("")
            post_url = post_node.css("::attr(href)").extract_first("")
            yield Request(url=parse.urljoin(response.url, post_url), meta={"front_image_url": image_url}, callback=self.parse_detial)

        # 提取出下一页的 url 并交给 Scrapy 进行下载
        next_url = response.css(".next.page-numbers::attr(href)").extract_first("")
        if next_url:
            yield Request(url=parse.urljoin(response.url, next_url), callback=self.parse)

    def parse_detial(self, response):
        # article_item = JobBoleArticleItem()
#
#         # 提取文章的具体字段
#
#         front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
#         # re_selector = response.xpath("/html/body/div[1]/div[3]/div[1]/div[1]/h1")  # firefox
#         # re_selector2 = response.xpath('//*[@id="post-113568"]/div[1]/h1/text()')  # chrome
#         title = response.xpath('//div[@class="entry-header"]/h1/text()').extract_first("")
#         create_date = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/text()').extract_first("").strip().replace("·","").strip()
#         praise_nums = response.xpath("//span[contains(@class, 'vote-post-up')]/h10/text()").extract_first(0)
#         fav_nums = response.xpath("//span[contains(@class, 'bookmark-btn')]/text()").extract_first(0)
#         match_re = re.match(r".*?(\d+).*", fav_nums)
#         if match_re:
#             fav_nums = int(match_re.group(1))
#         else:
#             fav_nums = 0
#
#         comment_nums = response.xpath("//a[@href='#article-comment']/span/text()").extract_first(0)
#         match_re = re.match(r".*?(\d+).*", comment_nums)
#         if match_re:
#             comment_nums = int(match_re.group(1))
#         else:
#             comment_nums = 0
#
#         content = response.xpath("//div[@class='entry']").extract_first("")
#
#         tag_list = response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
#
#         tag_list = [element for element in tag_list if not element.strip().endswith("评论")]
#         tags = ",".join(tag_list)
#
#
# # 通过css选择器提取字段
#         title_css = response.css(".entry-header h1::text").extract_first("")
#         create_date_css = response.css("p.entry-meta-hide-on-mobile::text").extract_first("").strip()
#         praise_nums_css = response.css(".vote-post-up h10::text").extract()[0]
#         fav_nums_css = response.css("span.bookmark-btn::text").extract_first(0)
#         match_re = re.match(r".*?(\d+).*", fav_nums_css)
#         if match_re:
#             fav_nums_css = int(match_re.group(1))
#         else:
#             fav_nums_css = 0
#
#         comment_nums_css = response.css("a[href='#article-comment'] span::text").extract_first(0)
#         match_re = re.match(r".*?(\d+).*", comment_nums_css)
#         if match_re:
#             comment_nums_css = int(match_re.group(1))
#         else:
#             comment_nums_css = 0
#
#         content_css = response.css('div.entry').extract_first("")
#
#         tag_list_css = response.css("p.entry-meta-hide-on-mobile a::text").extract()
#         tag_list_css = [element for element in tag_list_css if not element.strip().endswith("评论")]
#         tags_css = ",".join(tag_list)
#
#         article_item['url_object_id'] = get_md5(response.url)
#         article_item["title"] = title
#         article_item["url"] = response.url
#         try:
#             create_date = datetime.datetime.strptime(create_date, "%Y/%m/%d").date()
#         except Exception as e:
#             create_date = datetime.datetime.now().date()
#         article_item["create_date"] = create_date
#         article_item["front_image_url"] = [front_image_url]
#         article_item["praise_nums"] = praise_nums
#         article_item["comment_nums"] = comment_nums
#         article_item["fav_nums"] = fav_nums
#         article_item["tags"] = tags
#         article_item["content"] = content


        # 通过 item loader 加载 item
        # item_loader = ItemLoader(item=JobBoleArticleItem(), response=response)
        item_loader = ArticleItemLoader(item=JobBoleArticleItem(), response=response)
        front_image_url = response.meta.get("front_image_url", "")  # 文章封面图
        item_loader.add_css("title", ".entry-header h1::text")
        item_loader.add_value("url", response.url)
        item_loader.add_value("url_object_id", get_md5(response.url))
        item_loader.add_css("create_date", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_value("front_image_url", [front_image_url])
        item_loader.add_css("praise_nums", ".vote-post-up h10::text")
        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", "span.bookmark-btn::text")
        item_loader.add_css("tags", "p.entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content", "div.entry")

        article_item = item_loader.load_item()

        yield article_item
